#!/usr/bin/env python

import wsgiref.handlers
from google.appengine.ext import db
from google.appengine.ext import webapp
from google.appengine.api.labs import taskqueue
from schema import *

import logging

class FetchFinder(webapp.RequestHandler):
  def post(self):
    PAGESIZE = 5
    next = None
    computationid = long(self.request.get("computationid"))
    logging.debug("I'll parse fetches for computation %i." % computationid)
    computation = RankComputation.get_by_id(computationid)
    logging.debug("The computation key is %s." % str(computation.key()))
    bookmark = self.request.get("fetch_key")
    logging.debug("The fetch key is %s." % bookmark)
    
    fetches = Fetch.all().order("__key__")
    if bookmark:
      logging.debug("This is a continuation.")
      logging.debug("The last batch left off with %s." % bookmark)
      old_fetches = fetches.filter('__key__ >=', db.Key(bookmark))
      batch_of_fetches = old_fetches.fetch(PAGESIZE+1)
    else:
      logging.debug("This is the first batch.")
      batch_of_fetches = fetches.fetch(PAGESIZE+1)
    
    logging.debug("This batch has %i fetches." % len(batch_of_fetches))
    
    if len(batch_of_fetches) == PAGESIZE+1:
      logging.debug("There are enough fetches for another batch.")
      next = batch_of_fetches[-1]
      logging.debug("The next batch starts with fetch %s." % str(next.key()))
      task = taskqueue.Task(
        url='/find-fetches-to-parse',
        params={
          'computationid': computationid,
          'fetch_key': str(next.key()),
        }
      )
      task.add(queue_name='parse')
      batch_of_fetches = batch_of_fetches[:PAGESIZE]
    
    logging.debug("This batch has %i fetches." % len(batch_of_fetches))
    
    logging.debug("Now I'll make some tasks to find links.")
    for fetch in batch_of_fetches:
      if fetch.date < computation.date:
        graph = LinkGraphNode.all().filter('computationid =', computationid)
        node = graph.filter('docid =', fetch.docid).get()
        checksum = key_name_from_checksum(fetch.docid)
        logging.debug("Have I found links from %s before?" % checksum)
        if node is None:
          logging.debug("I'll make a task for fetch %s." % str(fetch.key()))
          task = taskqueue.Task(
            url='/find-links',
            params={
              'computationid': computationid,
              'fetch_key': str(fetch.key()),
            }
          )
          task.add(queue_name='parse')

def main():
  logging.getLogger().setLevel(logging.DEBUG)
  wsgiref.handlers.CGIHandler().run(webapp.WSGIApplication([
    ('/find-fetches-to-parse', FetchFinder),
  ]))


if __name__ == '__main__':
  main()
